Use hypercall batching where possible.
#ifdef CONFIG_SMP /* avoids "defined but not used" warnig */
static void flush_ldt(void *null)
{
- if (current->active_mm)
+ if (current->active_mm) {
load_LDT(¤t->active_mm->context);
+ flush_page_update_queue();
+ }
}
#endif
memcpy(new->ldt, old->ldt, old->size*LDT_ENTRY_SIZE);
make_pages_readonly(new->ldt, (new->size * LDT_ENTRY_SIZE) /
PAGE_SIZE);
+ flush_page_update_queue();
return 0;
}
set_call_gate(&default_ldt[0],lcall7);
set_call_gate(&default_ldt[4],lcall27);
__make_page_readonly(&default_ldt[0]);
+ xen_flush_page_update_queue();
/*
* Should be a barrier for any external CPU state.
#define QUEUE_SIZE 2048
#define pte_offset_kernel pte_offset
#else
-#define QUEUE_SIZE 1
+#define QUEUE_SIZE 128
#endif
static mmu_update_t update_queue[QUEUE_SIZE];
if ( unlikely(idx == QUEUE_SIZE) ) __flush_page_update_queue();
}
+static inline void increment_index_and_flush(void)
+{
+ idx++;
+ __flush_page_update_queue();
+}
+
void queue_l1_entry_update(pte_t *ptr, unsigned long val)
{
unsigned long flags;
spin_unlock_irqrestore(&update_lock, flags);
}
+/* queue and flush versions of the above */
+void xen_l1_entry_update(pte_t *ptr, unsigned long val)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+#if MMU_UPDATE_DEBUG > 3
+ DEBUG_disallow_pt_read((unsigned long)ptr);
+#endif
+ update_queue[idx].ptr = virt_to_machine(ptr);
+ update_queue[idx].val = val;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_l2_entry_update(pmd_t *ptr, unsigned long val)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = virt_to_machine(ptr);
+ update_queue[idx].val = val;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pt_switch(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = phys_to_machine(ptr);
+ update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_NEW_BASEPTR;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_tlb_flush(void)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_TLB_FLUSH;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_invlpg(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = MMU_EXTENDED_COMMAND;
+ update_queue[idx].ptr |= ptr & PAGE_MASK;
+ update_queue[idx].val = MMUEXT_INVLPG;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pgd_pin(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = phys_to_machine(ptr);
+ update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_PIN_L2_TABLE;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pgd_unpin(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = phys_to_machine(ptr);
+ update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_UNPIN_TABLE;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pte_pin(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = phys_to_machine(ptr);
+ update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_PIN_L1_TABLE;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_pte_unpin(unsigned long ptr)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = phys_to_machine(ptr);
+ update_queue[idx].ptr |= MMU_EXTENDED_COMMAND;
+ update_queue[idx].val = MMUEXT_UNPIN_TABLE;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_set_ldt(unsigned long ptr, unsigned long len)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = MMU_EXTENDED_COMMAND | ptr;
+ update_queue[idx].val = MMUEXT_SET_LDT | (len << MMUEXT_CMD_SHIFT);
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
+void xen_machphys_update(unsigned long mfn, unsigned long pfn)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&update_lock, flags);
+ update_queue[idx].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
+ update_queue[idx].val = pfn;
+ increment_index_and_flush();
+ spin_unlock_irqrestore(&update_lock, flags);
+}
+
#ifdef CONFIG_XEN_PHYSDEV_ACCESS
unsigned long allocate_empty_lowmem_region(unsigned long pages)
}
}
+/* NOTE: caller must call flush_page_update_queue() */
void __init wrprotect_bootpt(pgd_t *pgd, void *page, int set)
{
pmd_t *pmd;
pte_val_ma(*pte) | _PAGE_RW);
}
+/* NOTE: caller must call flush_page_update_queue() */
static void __init protect_bootpt_entries(pgd_t *spgd, pgd_t *dpgd, int set,
int pmdupdate, int pmdset)
{
}
}
}
- flush_page_update_queue();
}
static inline int is_kernel_text(unsigned long addr)
protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir,
1, 1, 1);
queue_pgd_pin(__pa(swapper_pg_dir));
- flush_page_update_queue();
load_cr3(swapper_pg_dir);
- __flush_tlb_all();
+ __flush_tlb_all(); /* implicit flush */
queue_pgd_unpin(__pa(start_info.pt_base));
- flush_page_update_queue();
protect_bootpt_entries((pgd_t *)start_info.pt_base, swapper_pg_dir,
0, 1, 0);
wrprotect_bootpt((pgd_t *)start_info.pt_base, swapper_pg_dir, 0);
+ flush_page_update_queue();
#ifdef CONFIG_X86_PAE
/*
if (pte) {
clear_page(pte);
__make_page_readonly(pte);
+ xen_flush_page_update_queue();
}
return pte;
}
#ifdef CONFIG_HIGHPTE
if (pte < highmem_start_page)
#endif
- __make_page_readonly(phys_to_virt(page_to_pseudophys(pte)));
+ {
+ __make_page_readonly(phys_to_virt(page_to_pseudophys(pte)));
+ flush_page_update_queue();
+ }
}
return pte;
}
{
unsigned long pfn = req->frame_and_sects[i] >> PAGE_SHIFT;
unsigned long mfn = phys_to_machine_mapping[pfn];
- queue_machphys_update(mfn, pfn);
+ xen_machphys_update(mfn, pfn);
}
break;
}
cpu_set(cpu, next->cpu_vm_mask);
/* Re-load page tables */
- load_cr3_noflush(next->pgd);
+ load_cr3(next->pgd);
/*
* load the LDT, if the LDT is different:
/* We were in lazy tlb mode and leave_mm disabled
* tlb flush IPI delivery. We must reload %cr3.
*/
- load_cr3_noflush(next->pgd);
+ load_cr3(next->pgd);
load_LDT_nolock(&next->context, cpu);
}
}
{
free_page((unsigned long)pte);
__make_page_writable(pte);
+ flush_page_update_queue();
}
static inline void pte_free(struct page *pte)
{
__make_page_writable(phys_to_virt(page_to_pseudophys(pte)));
__free_page(pte);
+ flush_page_update_queue();
}
}
#define set_pte(pteptr, pteval) (*(pteptr) = pteval)
#define set_pte_atomic(pteptr, pteval) (*(pteptr) = pteval)
#else
-#define set_pte(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low)
-#define set_pte_atomic(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low)
+#define set_pte(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low)
+#define set_pte_atomic(pteptr, pteval) xen_l1_entry_update(pteptr, (pteval).pte_low)
#endif
/*
* (pmds are folded into pgds so this doesn't get actually called,
* but the define is needed for a generic inline function.)
*/
-#define set_pmd(pmdptr, pmdval) queue_l2_entry_update((pmdptr), (pmdval).pmd)
+#define set_pmd(pmdptr, pmdval) xen_l2_entry_update((pmdptr), (pmdval).pmd)
#define set_pgd(pgdptr, pgdval) ((void)0)
#define pgd_page(pgd) \
pte_t pte = *ptep;
int ret = pte_dirty(pte);
if (ret)
- queue_l1_entry_update(ptep, pte_mkclean(pte).pte_low);
+ xen_l1_entry_update(ptep, pte_mkclean(pte).pte_low);
return ret;
}
pte_t pte = *ptep;
int ret = pte_young(pte);
if (ret)
- queue_l1_entry_update(ptep, pte_mkold(pte).pte_low);
+ xen_l1_entry_update(ptep, pte_mkold(pte).pte_low);
return ret;
}
{
pte_t pte = *ptep;
if (!pte_dirty(pte))
- queue_l1_entry_update(ptep, pte_mkdirty(pte).pte_low);
+ xen_l1_entry_update(ptep, pte_mkdirty(pte).pte_low);
}
/*
pmd_t p = *(xp); \
set_pmd(xp, __pmd(0)); \
__make_page_writable((void *)pmd_page_kernel(p)); \
- /* XXXcl queue */ \
+ xen_flush_page_update_queue(); \
} while (0)
#ifndef CONFIG_DISCONTIGMEM
#define update_mmu_cache(vma,address,pte) do { } while (0)
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
-#if 1
+#if 0
#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
do { \
if (__dirty) { \
queue_l1_entry_update((__ptep), (__entry).pte_low); \
- flush_tlb_page(__vma, __address); \
+ flush_tlb_page(__vma, __address); \
+ xen_flush_page_update_queue(); \
} \
} while (0)
#else
xen_flush_page_update_queue(); \
HYPERVISOR_update_va_mapping(address>>PAGE_SHIFT, entry, UVMF_INVLPG); \
} else { \
- queue_l1_entry_update((__ptep), (__entry).pte_low); \
- xen_flush_page_update_queue(); \
+ xen_l1_entry_update((__ptep), (__entry).pte_low); \
} \
} \
} while (0)
#define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low })
#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
+/* NOTE: make_page* callers must call flush_page_update_queue() */
static inline void __make_page_readonly(void *va)
{
pgd_t *pgd = pgd_offset_k((unsigned long)va);
if ( (unsigned long)va >= VMALLOC_START )
__make_page_readonly(machine_to_virt(
*(unsigned long *)pte&PAGE_MASK));
- /* XXXcl queue */
}
static inline void make_page_writable(void *va)
if ( (unsigned long)va >= VMALLOC_START )
__make_page_writable(machine_to_virt(
*(unsigned long *)pte&PAGE_MASK));
- /* XXXcl queue */
}
static inline void make_pages_readonly(void *va, unsigned int nr)
make_page_readonly(va);
va = (void *)((unsigned long)va + PAGE_SIZE);
}
- /* XXXcl queue */
}
static inline void make_pages_writable(void *va, unsigned int nr)
make_page_writable(va);
va = (void *)((unsigned long)va + PAGE_SIZE);
}
- /* XXXcl queue */
}
static inline unsigned long arbitrary_virt_to_phys(void *va)
}
#define load_cr3(pgdir) do { \
- queue_pt_switch(__pa(pgdir)); \
- flush_page_update_queue(); \
- cur_pgd = pgdir; /* XXXsmp */ \
-} while (/* CONSTCOND */0)
-
-#define load_cr3_noflush(pgdir) do { \
queue_pt_switch(__pa(pgdir)); \
cur_pgd = pgdir; /* XXXsmp */ \
} while (/* CONSTCOND */0)
#include <asm/processor.h>
#define __flush_tlb() do { \
- queue_tlb_flush(); \
- xen_flush_page_update_queue(); \
+ xen_tlb_flush(); \
} while (/*CONSTCOND*/0)
/*
*/
#define __flush_tlb_global() \
do { \
- queue_tlb_flush(); \
- xen_flush_page_update_queue(); \
+ xen_tlb_flush(); \
} while (0)
extern unsigned long pgkern_mask;
#define cpu_has_invlpg (boot_cpu_data.x86 > 3)
#define __flush_tlb_single(addr) do { \
- queue_invlpg(addr); \
- xen_flush_page_update_queue(); \
+ xen_invlpg(addr); \
} while (/* CONSTCOND */0)
# define __flush_tlb_one(addr) __flush_tlb_single(addr)
void queue_pte_unpin(unsigned long ptr);
void queue_set_ldt(unsigned long ptr, unsigned long bytes);
void queue_machphys_update(unsigned long mfn, unsigned long pfn);
+void xen_l1_entry_update(pte_t *ptr, unsigned long val);
+void xen_l2_entry_update(pmd_t *ptr, unsigned long val);
+void xen_pt_switch(unsigned long ptr);
+void xen_tlb_flush(void);
+void xen_invlpg(unsigned long ptr);
+void xen_pgd_pin(unsigned long ptr);
+void xen_pgd_unpin(unsigned long ptr);
+void xen_pte_pin(unsigned long ptr);
+void xen_pte_unpin(unsigned long ptr);
+void xen_set_ldt(unsigned long ptr, unsigned long bytes);
+void xen_machphys_update(unsigned long mfn, unsigned long pfn);
#define MMU_UPDATE_DEBUG 0
#if MMU_UPDATE_DEBUG > 0
projects / xen.git / commitdiff